{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def swish(x):\n",
    "    return x*tf.nn.sigmoid(x)\n",
    "\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "def selu(x):\n",
    "    with tf.name_scope('elu') as scope:\n",
    "        alpha = 1.673263242354377284170429916717\n",
    "        scale = 1.050700987355480493419349852946\n",
    "        return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x))\n",
    "    \n",
    "def activation(x):\n",
    "    return swish(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#初始化\n",
    "def initialize(shape, stddev=0.1):\n",
    "    return tf.truncated_normal(shape, stddev=stddev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/train-images-idx3-ubyte.gz\n",
      "Extracting ./data/train-labels-idx1-ubyte.gz\n",
      "Extracting ./data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = './data/'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个非常非常简陋的模型\n",
    "每个输入数据为784维的向量\n",
    "输出为十维的向量，表示的是介于0到9的数字，标签0将表示成([1,0,0,0,0,0,0,0,0,0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "# Create the model\n",
    "L1_units_count = 100\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "#60000 / \n",
    "epoch_steps =  tf.to_int64(tf.div(60000, tf.shape(x)[0]))\n",
    "#声明一个计数器\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch = global_step//epoch_steps\n",
    "decay_times = current_epoch #//2\n",
    "current_learning_rate=tf.multiply(init_learning_rate, tf.pow(0.575, tf.to_float(decay_times)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W1 = tf.Variable(initialize([784, L1_units_count], stddev=np.sqrt(2/784)))\n",
    "b1 = tf.Variable(tf.constant(0.001, shape=[L1_units_count]))\n",
    "logits1 = tf.matmul(x, W1) + b1\n",
    "output1 = activation(logits1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "L2_units_count = 10\n",
    "W2 = tf.Variable(initialize([L1_units_count, L2_units_count], stddev=np.sqrt(2/L1_units_count)))\n",
    "b2 = tf.Variable(tf.constant(0.001, shape=[L2_units_count]))\n",
    "logits2 = tf.matmul(output1, W2) + b2\n",
    "y = logits2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#4e-5,用于平衡l2_loss的值，作为惩罚因子\n",
    "l2_loss = tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2)\n",
    "total_loss = cross_entropy + 4e-5*l2_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#optimize =  tf.train.AdamOptimizer(current_learning_rate)\n",
    "#gradients = optimize.compute_gradients(total_loss)\n",
    "#train_step = optimize.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(current_learning_rate).minimize(total_loss, global_step=global_step)\n",
    " # Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 0.050401, l2_loss: 494.990448, total_loss: 0.070201\n",
      "0.9578\n",
      "step 200, entropy loss: 0.055138, l2_loss: 588.371887, total_loss: 0.078673\n",
      "0.9555\n",
      "step 300, entropy loss: 0.091787, l2_loss: 679.626953, total_loss: 0.118972\n",
      "0.9553\n",
      "step 400, entropy loss: 0.066009, l2_loss: 745.362183, total_loss: 0.095823\n",
      "0.9534\n",
      "step 500, entropy loss: 0.099501, l2_loss: 740.929504, total_loss: 0.129138\n",
      "0.9723\n",
      "step 600, entropy loss: 0.052568, l2_loss: 728.320679, total_loss: 0.081701\n",
      "0.9674\n",
      "step 700, entropy loss: 0.025217, l2_loss: 725.811768, total_loss: 0.054250\n",
      "0.9695\n",
      "step 800, entropy loss: 0.045033, l2_loss: 727.098938, total_loss: 0.074117\n",
      "0.9721\n",
      "step 900, entropy loss: 0.062409, l2_loss: 738.159302, total_loss: 0.091935\n",
      "0.9724\n",
      "step 1000, entropy loss: 0.061159, l2_loss: 744.231567, total_loss: 0.090928\n",
      "0.9741\n",
      "step 1100, entropy loss: 0.052582, l2_loss: 734.469910, total_loss: 0.081960\n",
      "0.9771\n",
      "step 1200, entropy loss: 0.070849, l2_loss: 724.203186, total_loss: 0.099818\n",
      "0.978\n",
      "step 1300, entropy loss: 0.036845, l2_loss: 717.103088, total_loss: 0.065529\n",
      "0.976\n",
      "step 1400, entropy loss: 0.049922, l2_loss: 712.855774, total_loss: 0.078436\n",
      "0.9738\n",
      "step 1500, entropy loss: 0.008516, l2_loss: 710.792053, total_loss: 0.036948\n",
      "0.9803\n",
      "step 1600, entropy loss: 0.024212, l2_loss: 702.601807, total_loss: 0.052316\n",
      "0.9769\n",
      "step 1700, entropy loss: 0.004703, l2_loss: 694.423523, total_loss: 0.032480\n",
      "0.9792\n",
      "step 1800, entropy loss: 0.045003, l2_loss: 687.885010, total_loss: 0.072519\n",
      "0.9781\n",
      "step 1900, entropy loss: 0.008855, l2_loss: 679.900024, total_loss: 0.036051\n",
      "0.9811\n",
      "step 2000, entropy loss: 0.008920, l2_loss: 674.818909, total_loss: 0.035913\n",
      "0.9778\n",
      "step 2100, entropy loss: 0.035619, l2_loss: 667.846191, total_loss: 0.062333\n",
      "0.981\n",
      "step 2200, entropy loss: 0.017402, l2_loss: 662.614990, total_loss: 0.043906\n",
      "0.9788\n",
      "step 2300, entropy loss: 0.016603, l2_loss: 657.627808, total_loss: 0.042908\n",
      "0.9809\n",
      "step 2400, entropy loss: 0.021087, l2_loss: 652.429810, total_loss: 0.047184\n",
      "0.9812\n",
      "step 2500, entropy loss: 0.040294, l2_loss: 647.893372, total_loss: 0.066210\n",
      "0.9804\n",
      "step 2600, entropy loss: 0.005146, l2_loss: 642.676575, total_loss: 0.030853\n",
      "0.9804\n",
      "step 2700, entropy loss: 0.031278, l2_loss: 637.245544, total_loss: 0.056768\n",
      "0.9804\n",
      "step 2800, entropy loss: 0.065469, l2_loss: 632.479980, total_loss: 0.090768\n",
      "0.9811\n",
      "step 2900, entropy loss: 0.013236, l2_loss: 629.627808, total_loss: 0.038421\n",
      "0.9808\n",
      "step 3000, entropy loss: 0.037773, l2_loss: 626.786987, total_loss: 0.062844\n",
      "0.982\n"
     ]
    }
   ],
   "source": [
    "# 每次取训练数据中的100样本来测试，让模型循环训练3000次\n",
    " \n",
    "for step in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    lr = 1e-2\n",
    "    _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "    sess.run([train_step, cross_entropy, l2_loss, total_loss, current_learning_rate], \\\n",
    "            feed_dict={x: batch_xs, y_: batch_ys, init_learning_rate:lr})\n",
    "  \n",
    "    if (step+1) % 100 == 0:\n",
    "        print('step %d, entropy loss: %f, l2_loss: %f, total_loss: %f' %\n",
    "              (step+1, loss, l2_loss_value, total_loss_value))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
